{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf \n",
    "import re\n",
    "import torch\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf_path = 'weights/pubmed_pmc_470k/biobert_model.ckpt'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "init_vars = tf.train.list_variables(tf_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "excluded = ['BERTAdam','_power','global_step']\n",
    "init_vars = list(filter(lambda x:all([True if e not in x[0] else False for e in excluded]),init_vars))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('bert/embeddings/LayerNorm/beta', [768]),\n",
       " ('bert/embeddings/LayerNorm/gamma', [768]),\n",
       " ('bert/embeddings/position_embeddings', [512, 768]),\n",
       " ('bert/embeddings/token_type_embeddings', [2, 768]),\n",
       " ('bert/embeddings/word_embeddings', [28996, 768]),\n",
       " ('bert/encoder/layer_0/attention/output/LayerNorm/beta', [768]),\n",
       " ('bert/encoder/layer_0/attention/output/LayerNorm/gamma', [768]),\n",
       " ('bert/encoder/layer_0/attention/output/dense/bias', [768]),\n",
       " ('bert/encoder/layer_0/attention/output/dense/kernel', [768, 768]),\n",
       " ('bert/encoder/layer_0/attention/self/key/bias', [768]),\n",
       " ('bert/encoder/layer_0/attention/self/key/kernel', [768, 768]),\n",
       " ('bert/encoder/layer_0/attention/self/query/bias', [768]),\n",
       " ('bert/encoder/layer_0/attention/self/query/kernel', [768, 768]),\n",
       " ('bert/encoder/layer_0/attention/self/value/bias', [768]),\n",
       " ('bert/encoder/layer_0/attention/self/value/kernel', [768, 768]),\n",
       " ('bert/encoder/layer_0/intermediate/dense/bias', [3072]),\n",
       " ('bert/encoder/layer_0/intermediate/dense/kernel', [768, 3072]),\n",
       " ('bert/encoder/layer_0/output/LayerNorm/beta', [768]),\n",
       " ('bert/encoder/layer_0/output/LayerNorm/gamma', [768]),\n",
       " ('bert/encoder/layer_0/output/dense/bias', [768]),\n",
       " ('bert/encoder/layer_0/output/dense/kernel', [3072, 768]),\n",
       " ('bert/encoder/layer_1/attention/output/LayerNorm/beta', [768]),\n",
       " ('bert/encoder/layer_1/attention/output/LayerNorm/gamma', [768]),\n",
       " ('bert/encoder/layer_1/attention/output/dense/bias', [768]),\n",
       " ('bert/encoder/layer_1/attention/output/dense/kernel', [768, 768]),\n",
       " ('bert/encoder/layer_1/attention/self/key/bias', [768]),\n",
       " ('bert/encoder/layer_1/attention/self/key/kernel', [768, 768]),\n",
       " ('bert/encoder/layer_1/attention/self/query/bias', [768]),\n",
       " ('bert/encoder/layer_1/attention/self/query/kernel', [768, 768]),\n",
       " ('bert/encoder/layer_1/attention/self/value/bias', [768]),\n",
       " ('bert/encoder/layer_1/attention/self/value/kernel', [768, 768]),\n",
       " ('bert/encoder/layer_1/intermediate/dense/bias', [3072]),\n",
       " ('bert/encoder/layer_1/intermediate/dense/kernel', [768, 3072]),\n",
       " ('bert/encoder/layer_1/output/LayerNorm/beta', [768]),\n",
       " ('bert/encoder/layer_1/output/LayerNorm/gamma', [768]),\n",
       " ('bert/encoder/layer_1/output/dense/bias', [768]),\n",
       " ('bert/encoder/layer_1/output/dense/kernel', [3072, 768]),\n",
       " ('bert/encoder/layer_10/attention/output/LayerNorm/beta', [768]),\n",
       " ('bert/encoder/layer_10/attention/output/LayerNorm/gamma', [768]),\n",
       " ('bert/encoder/layer_10/attention/output/dense/bias', [768]),\n",
       " ('bert/encoder/layer_10/attention/output/dense/kernel', [768, 768]),\n",
       " ('bert/encoder/layer_10/attention/self/key/bias', [768]),\n",
       " ('bert/encoder/layer_10/attention/self/key/kernel', [768, 768]),\n",
       " ('bert/encoder/layer_10/attention/self/query/bias', [768]),\n",
       " ('bert/encoder/layer_10/attention/self/query/kernel', [768, 768]),\n",
       " ('bert/encoder/layer_10/attention/self/value/bias', [768]),\n",
       " ('bert/encoder/layer_10/attention/self/value/kernel', [768, 768]),\n",
       " ('bert/encoder/layer_10/intermediate/dense/bias', [3072]),\n",
       " ('bert/encoder/layer_10/intermediate/dense/kernel', [768, 3072]),\n",
       " ('bert/encoder/layer_10/output/LayerNorm/beta', [768]),\n",
       " ('bert/encoder/layer_10/output/LayerNorm/gamma', [768]),\n",
       " ('bert/encoder/layer_10/output/dense/bias', [768]),\n",
       " ('bert/encoder/layer_10/output/dense/kernel', [3072, 768]),\n",
       " ('bert/encoder/layer_11/attention/output/LayerNorm/beta', [768]),\n",
       " ('bert/encoder/layer_11/attention/output/LayerNorm/gamma', [768]),\n",
       " ('bert/encoder/layer_11/attention/output/dense/bias', [768]),\n",
       " ('bert/encoder/layer_11/attention/output/dense/kernel', [768, 768]),\n",
       " ('bert/encoder/layer_11/attention/self/key/bias', [768]),\n",
       " ('bert/encoder/layer_11/attention/self/key/kernel', [768, 768]),\n",
       " ('bert/encoder/layer_11/attention/self/query/bias', [768]),\n",
       " ('bert/encoder/layer_11/attention/self/query/kernel', [768, 768]),\n",
       " ('bert/encoder/layer_11/attention/self/value/bias', [768]),\n",
       " ('bert/encoder/layer_11/attention/self/value/kernel', [768, 768]),\n",
       " ('bert/encoder/layer_11/intermediate/dense/bias', [3072]),\n",
       " ('bert/encoder/layer_11/intermediate/dense/kernel', [768, 3072]),\n",
       " ('bert/encoder/layer_11/output/LayerNorm/beta', [768]),\n",
       " ('bert/encoder/layer_11/output/LayerNorm/gamma', [768]),\n",
       " ('bert/encoder/layer_11/output/dense/bias', [768]),\n",
       " ('bert/encoder/layer_11/output/dense/kernel', [3072, 768]),\n",
       " ('bert/encoder/layer_2/attention/output/LayerNorm/beta', [768]),\n",
       " ('bert/encoder/layer_2/attention/output/LayerNorm/gamma', [768]),\n",
       " ('bert/encoder/layer_2/attention/output/dense/bias', [768]),\n",
       " ('bert/encoder/layer_2/attention/output/dense/kernel', [768, 768]),\n",
       " ('bert/encoder/layer_2/attention/self/key/bias', [768]),\n",
       " ('bert/encoder/layer_2/attention/self/key/kernel', [768, 768]),\n",
       " ('bert/encoder/layer_2/attention/self/query/bias', [768]),\n",
       " ('bert/encoder/layer_2/attention/self/query/kernel', [768, 768]),\n",
       " ('bert/encoder/layer_2/attention/self/value/bias', [768]),\n",
       " ('bert/encoder/layer_2/attention/self/value/kernel', [768, 768]),\n",
       " ('bert/encoder/layer_2/intermediate/dense/bias', [3072]),\n",
       " ('bert/encoder/layer_2/intermediate/dense/kernel', [768, 3072]),\n",
       " ('bert/encoder/layer_2/output/LayerNorm/beta', [768]),\n",
       " ('bert/encoder/layer_2/output/LayerNorm/gamma', [768]),\n",
       " ('bert/encoder/layer_2/output/dense/bias', [768]),\n",
       " ('bert/encoder/layer_2/output/dense/kernel', [3072, 768]),\n",
       " ('bert/encoder/layer_3/attention/output/LayerNorm/beta', [768]),\n",
       " ('bert/encoder/layer_3/attention/output/LayerNorm/gamma', [768]),\n",
       " ('bert/encoder/layer_3/attention/output/dense/bias', [768]),\n",
       " ('bert/encoder/layer_3/attention/output/dense/kernel', [768, 768]),\n",
       " ('bert/encoder/layer_3/attention/self/key/bias', [768]),\n",
       " ('bert/encoder/layer_3/attention/self/key/kernel', [768, 768]),\n",
       " ('bert/encoder/layer_3/attention/self/query/bias', [768]),\n",
       " ('bert/encoder/layer_3/attention/self/query/kernel', [768, 768]),\n",
       " ('bert/encoder/layer_3/attention/self/value/bias', [768]),\n",
       " ('bert/encoder/layer_3/attention/self/value/kernel', [768, 768]),\n",
       " ('bert/encoder/layer_3/intermediate/dense/bias', [3072]),\n",
       " ('bert/encoder/layer_3/intermediate/dense/kernel', [768, 3072]),\n",
       " ('bert/encoder/layer_3/output/LayerNorm/beta', [768]),\n",
       " ('bert/encoder/layer_3/output/LayerNorm/gamma', [768]),\n",
       " ('bert/encoder/layer_3/output/dense/bias', [768]),\n",
       " ('bert/encoder/layer_3/output/dense/kernel', [3072, 768]),\n",
       " ('bert/encoder/layer_4/attention/output/LayerNorm/beta', [768]),\n",
       " ('bert/encoder/layer_4/attention/output/LayerNorm/gamma', [768]),\n",
       " ('bert/encoder/layer_4/attention/output/dense/bias', [768]),\n",
       " ('bert/encoder/layer_4/attention/output/dense/kernel', [768, 768]),\n",
       " ('bert/encoder/layer_4/attention/self/key/bias', [768]),\n",
       " ('bert/encoder/layer_4/attention/self/key/kernel', [768, 768]),\n",
       " ('bert/encoder/layer_4/attention/self/query/bias', [768]),\n",
       " ('bert/encoder/layer_4/attention/self/query/kernel', [768, 768]),\n",
       " ('bert/encoder/layer_4/attention/self/value/bias', [768]),\n",
       " ('bert/encoder/layer_4/attention/self/value/kernel', [768, 768]),\n",
       " ('bert/encoder/layer_4/intermediate/dense/bias', [3072]),\n",
       " ('bert/encoder/layer_4/intermediate/dense/kernel', [768, 3072]),\n",
       " ('bert/encoder/layer_4/output/LayerNorm/beta', [768]),\n",
       " ('bert/encoder/layer_4/output/LayerNorm/gamma', [768]),\n",
       " ('bert/encoder/layer_4/output/dense/bias', [768]),\n",
       " ('bert/encoder/layer_4/output/dense/kernel', [3072, 768]),\n",
       " ('bert/encoder/layer_5/attention/output/LayerNorm/beta', [768]),\n",
       " ('bert/encoder/layer_5/attention/output/LayerNorm/gamma', [768]),\n",
       " ('bert/encoder/layer_5/attention/output/dense/bias', [768]),\n",
       " ('bert/encoder/layer_5/attention/output/dense/kernel', [768, 768]),\n",
       " ('bert/encoder/layer_5/attention/self/key/bias', [768]),\n",
       " ('bert/encoder/layer_5/attention/self/key/kernel', [768, 768]),\n",
       " ('bert/encoder/layer_5/attention/self/query/bias', [768]),\n",
       " ('bert/encoder/layer_5/attention/self/query/kernel', [768, 768]),\n",
       " ('bert/encoder/layer_5/attention/self/value/bias', [768]),\n",
       " ('bert/encoder/layer_5/attention/self/value/kernel', [768, 768]),\n",
       " ('bert/encoder/layer_5/intermediate/dense/bias', [3072]),\n",
       " ('bert/encoder/layer_5/intermediate/dense/kernel', [768, 3072]),\n",
       " ('bert/encoder/layer_5/output/LayerNorm/beta', [768]),\n",
       " ('bert/encoder/layer_5/output/LayerNorm/gamma', [768]),\n",
       " ('bert/encoder/layer_5/output/dense/bias', [768]),\n",
       " ('bert/encoder/layer_5/output/dense/kernel', [3072, 768]),\n",
       " ('bert/encoder/layer_6/attention/output/LayerNorm/beta', [768]),\n",
       " ('bert/encoder/layer_6/attention/output/LayerNorm/gamma', [768]),\n",
       " ('bert/encoder/layer_6/attention/output/dense/bias', [768]),\n",
       " ('bert/encoder/layer_6/attention/output/dense/kernel', [768, 768]),\n",
       " ('bert/encoder/layer_6/attention/self/key/bias', [768]),\n",
       " ('bert/encoder/layer_6/attention/self/key/kernel', [768, 768]),\n",
       " ('bert/encoder/layer_6/attention/self/query/bias', [768]),\n",
       " ('bert/encoder/layer_6/attention/self/query/kernel', [768, 768]),\n",
       " ('bert/encoder/layer_6/attention/self/value/bias', [768]),\n",
       " ('bert/encoder/layer_6/attention/self/value/kernel', [768, 768]),\n",
       " ('bert/encoder/layer_6/intermediate/dense/bias', [3072]),\n",
       " ('bert/encoder/layer_6/intermediate/dense/kernel', [768, 3072]),\n",
       " ('bert/encoder/layer_6/output/LayerNorm/beta', [768]),\n",
       " ('bert/encoder/layer_6/output/LayerNorm/gamma', [768]),\n",
       " ('bert/encoder/layer_6/output/dense/bias', [768]),\n",
       " ('bert/encoder/layer_6/output/dense/kernel', [3072, 768]),\n",
       " ('bert/encoder/layer_7/attention/output/LayerNorm/beta', [768]),\n",
       " ('bert/encoder/layer_7/attention/output/LayerNorm/gamma', [768]),\n",
       " ('bert/encoder/layer_7/attention/output/dense/bias', [768]),\n",
       " ('bert/encoder/layer_7/attention/output/dense/kernel', [768, 768]),\n",
       " ('bert/encoder/layer_7/attention/self/key/bias', [768]),\n",
       " ('bert/encoder/layer_7/attention/self/key/kernel', [768, 768]),\n",
       " ('bert/encoder/layer_7/attention/self/query/bias', [768]),\n",
       " ('bert/encoder/layer_7/attention/self/query/kernel', [768, 768]),\n",
       " ('bert/encoder/layer_7/attention/self/value/bias', [768]),\n",
       " ('bert/encoder/layer_7/attention/self/value/kernel', [768, 768]),\n",
       " ('bert/encoder/layer_7/intermediate/dense/bias', [3072]),\n",
       " ('bert/encoder/layer_7/intermediate/dense/kernel', [768, 3072]),\n",
       " ('bert/encoder/layer_7/output/LayerNorm/beta', [768]),\n",
       " ('bert/encoder/layer_7/output/LayerNorm/gamma', [768]),\n",
       " ('bert/encoder/layer_7/output/dense/bias', [768]),\n",
       " ('bert/encoder/layer_7/output/dense/kernel', [3072, 768]),\n",
       " ('bert/encoder/layer_8/attention/output/LayerNorm/beta', [768]),\n",
       " ('bert/encoder/layer_8/attention/output/LayerNorm/gamma', [768]),\n",
       " ('bert/encoder/layer_8/attention/output/dense/bias', [768]),\n",
       " ('bert/encoder/layer_8/attention/output/dense/kernel', [768, 768]),\n",
       " ('bert/encoder/layer_8/attention/self/key/bias', [768]),\n",
       " ('bert/encoder/layer_8/attention/self/key/kernel', [768, 768]),\n",
       " ('bert/encoder/layer_8/attention/self/query/bias', [768]),\n",
       " ('bert/encoder/layer_8/attention/self/query/kernel', [768, 768]),\n",
       " ('bert/encoder/layer_8/attention/self/value/bias', [768]),\n",
       " ('bert/encoder/layer_8/attention/self/value/kernel', [768, 768]),\n",
       " ('bert/encoder/layer_8/intermediate/dense/bias', [3072]),\n",
       " ('bert/encoder/layer_8/intermediate/dense/kernel', [768, 3072]),\n",
       " ('bert/encoder/layer_8/output/LayerNorm/beta', [768]),\n",
       " ('bert/encoder/layer_8/output/LayerNorm/gamma', [768]),\n",
       " ('bert/encoder/layer_8/output/dense/bias', [768]),\n",
       " ('bert/encoder/layer_8/output/dense/kernel', [3072, 768]),\n",
       " ('bert/encoder/layer_9/attention/output/LayerNorm/beta', [768]),\n",
       " ('bert/encoder/layer_9/attention/output/LayerNorm/gamma', [768]),\n",
       " ('bert/encoder/layer_9/attention/output/dense/bias', [768]),\n",
       " ('bert/encoder/layer_9/attention/output/dense/kernel', [768, 768]),\n",
       " ('bert/encoder/layer_9/attention/self/key/bias', [768]),\n",
       " ('bert/encoder/layer_9/attention/self/key/kernel', [768, 768]),\n",
       " ('bert/encoder/layer_9/attention/self/query/bias', [768]),\n",
       " ('bert/encoder/layer_9/attention/self/query/kernel', [768, 768]),\n",
       " ('bert/encoder/layer_9/attention/self/value/bias', [768]),\n",
       " ('bert/encoder/layer_9/attention/self/value/kernel', [768, 768]),\n",
       " ('bert/encoder/layer_9/intermediate/dense/bias', [3072]),\n",
       " ('bert/encoder/layer_9/intermediate/dense/kernel', [768, 3072]),\n",
       " ('bert/encoder/layer_9/output/LayerNorm/beta', [768]),\n",
       " ('bert/encoder/layer_9/output/LayerNorm/gamma', [768]),\n",
       " ('bert/encoder/layer_9/output/dense/bias', [768]),\n",
       " ('bert/encoder/layer_9/output/dense/kernel', [3072, 768]),\n",
       " ('bert/pooler/dense/bias', [768]),\n",
       " ('bert/pooler/dense/kernel', [768, 768]),\n",
       " ('cls/predictions/output_bias', [28996]),\n",
       " ('cls/predictions/transform/LayerNorm/beta', [768]),\n",
       " ('cls/predictions/transform/LayerNorm/gamma', [768]),\n",
       " ('cls/predictions/transform/dense/bias', [768]),\n",
       " ('cls/predictions/transform/dense/kernel', [768, 768]),\n",
       " ('cls/seq_relationship/output_bias', [2]),\n",
       " ('cls/seq_relationship/output_weights', [2, 768])]"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "init_vars"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading TF weight bert/embeddings/LayerNorm/beta with shape [768]\n",
      "Loading TF weight bert/embeddings/LayerNorm/gamma with shape [768]\n",
      "Loading TF weight bert/embeddings/position_embeddings with shape [512, 768]\n",
      "Loading TF weight bert/embeddings/token_type_embeddings with shape [2, 768]\n",
      "Loading TF weight bert/embeddings/word_embeddings with shape [28996, 768]\n",
      "Loading TF weight bert/encoder/layer_0/attention/output/LayerNorm/beta with shape [768]\n",
      "Loading TF weight bert/encoder/layer_0/attention/output/LayerNorm/gamma with shape [768]\n",
      "Loading TF weight bert/encoder/layer_0/attention/output/dense/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_0/attention/output/dense/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_0/attention/self/key/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_0/attention/self/key/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_0/attention/self/query/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_0/attention/self/query/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_0/attention/self/value/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_0/attention/self/value/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_0/intermediate/dense/bias with shape [3072]\n",
      "Loading TF weight bert/encoder/layer_0/intermediate/dense/kernel with shape [768, 3072]\n",
      "Loading TF weight bert/encoder/layer_0/output/LayerNorm/beta with shape [768]\n",
      "Loading TF weight bert/encoder/layer_0/output/LayerNorm/gamma with shape [768]\n",
      "Loading TF weight bert/encoder/layer_0/output/dense/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_0/output/dense/kernel with shape [3072, 768]\n",
      "Loading TF weight bert/encoder/layer_1/attention/output/LayerNorm/beta with shape [768]\n",
      "Loading TF weight bert/encoder/layer_1/attention/output/LayerNorm/gamma with shape [768]\n",
      "Loading TF weight bert/encoder/layer_1/attention/output/dense/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_1/attention/output/dense/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_1/attention/self/key/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_1/attention/self/key/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_1/attention/self/query/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_1/attention/self/query/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_1/attention/self/value/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_1/attention/self/value/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_1/intermediate/dense/bias with shape [3072]\n",
      "Loading TF weight bert/encoder/layer_1/intermediate/dense/kernel with shape [768, 3072]\n",
      "Loading TF weight bert/encoder/layer_1/output/LayerNorm/beta with shape [768]\n",
      "Loading TF weight bert/encoder/layer_1/output/LayerNorm/gamma with shape [768]\n",
      "Loading TF weight bert/encoder/layer_1/output/dense/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_1/output/dense/kernel with shape [3072, 768]\n",
      "Loading TF weight bert/encoder/layer_10/attention/output/LayerNorm/beta with shape [768]\n",
      "Loading TF weight bert/encoder/layer_10/attention/output/LayerNorm/gamma with shape [768]\n",
      "Loading TF weight bert/encoder/layer_10/attention/output/dense/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_10/attention/output/dense/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_10/attention/self/key/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_10/attention/self/key/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_10/attention/self/query/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_10/attention/self/query/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_10/attention/self/value/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_10/attention/self/value/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_10/intermediate/dense/bias with shape [3072]\n",
      "Loading TF weight bert/encoder/layer_10/intermediate/dense/kernel with shape [768, 3072]\n",
      "Loading TF weight bert/encoder/layer_10/output/LayerNorm/beta with shape [768]\n",
      "Loading TF weight bert/encoder/layer_10/output/LayerNorm/gamma with shape [768]\n",
      "Loading TF weight bert/encoder/layer_10/output/dense/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_10/output/dense/kernel with shape [3072, 768]\n",
      "Loading TF weight bert/encoder/layer_11/attention/output/LayerNorm/beta with shape [768]\n",
      "Loading TF weight bert/encoder/layer_11/attention/output/LayerNorm/gamma with shape [768]\n",
      "Loading TF weight bert/encoder/layer_11/attention/output/dense/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_11/attention/output/dense/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_11/attention/self/key/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_11/attention/self/key/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_11/attention/self/query/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_11/attention/self/query/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_11/attention/self/value/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_11/attention/self/value/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_11/intermediate/dense/bias with shape [3072]\n",
      "Loading TF weight bert/encoder/layer_11/intermediate/dense/kernel with shape [768, 3072]\n",
      "Loading TF weight bert/encoder/layer_11/output/LayerNorm/beta with shape [768]\n",
      "Loading TF weight bert/encoder/layer_11/output/LayerNorm/gamma with shape [768]\n",
      "Loading TF weight bert/encoder/layer_11/output/dense/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_11/output/dense/kernel with shape [3072, 768]\n",
      "Loading TF weight bert/encoder/layer_2/attention/output/LayerNorm/beta with shape [768]\n",
      "Loading TF weight bert/encoder/layer_2/attention/output/LayerNorm/gamma with shape [768]\n",
      "Loading TF weight bert/encoder/layer_2/attention/output/dense/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_2/attention/output/dense/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_2/attention/self/key/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_2/attention/self/key/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_2/attention/self/query/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_2/attention/self/query/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_2/attention/self/value/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_2/attention/self/value/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_2/intermediate/dense/bias with shape [3072]\n",
      "Loading TF weight bert/encoder/layer_2/intermediate/dense/kernel with shape [768, 3072]\n",
      "Loading TF weight bert/encoder/layer_2/output/LayerNorm/beta with shape [768]\n",
      "Loading TF weight bert/encoder/layer_2/output/LayerNorm/gamma with shape [768]\n",
      "Loading TF weight bert/encoder/layer_2/output/dense/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_2/output/dense/kernel with shape [3072, 768]\n",
      "Loading TF weight bert/encoder/layer_3/attention/output/LayerNorm/beta with shape [768]\n",
      "Loading TF weight bert/encoder/layer_3/attention/output/LayerNorm/gamma with shape [768]\n",
      "Loading TF weight bert/encoder/layer_3/attention/output/dense/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_3/attention/output/dense/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_3/attention/self/key/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_3/attention/self/key/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_3/attention/self/query/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_3/attention/self/query/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_3/attention/self/value/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_3/attention/self/value/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_3/intermediate/dense/bias with shape [3072]\n",
      "Loading TF weight bert/encoder/layer_3/intermediate/dense/kernel with shape [768, 3072]\n",
      "Loading TF weight bert/encoder/layer_3/output/LayerNorm/beta with shape [768]\n",
      "Loading TF weight bert/encoder/layer_3/output/LayerNorm/gamma with shape [768]\n",
      "Loading TF weight bert/encoder/layer_3/output/dense/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_3/output/dense/kernel with shape [3072, 768]\n",
      "Loading TF weight bert/encoder/layer_4/attention/output/LayerNorm/beta with shape [768]\n",
      "Loading TF weight bert/encoder/layer_4/attention/output/LayerNorm/gamma with shape [768]\n",
      "Loading TF weight bert/encoder/layer_4/attention/output/dense/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_4/attention/output/dense/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_4/attention/self/key/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_4/attention/self/key/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_4/attention/self/query/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_4/attention/self/query/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_4/attention/self/value/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_4/attention/self/value/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_4/intermediate/dense/bias with shape [3072]\n",
      "Loading TF weight bert/encoder/layer_4/intermediate/dense/kernel with shape [768, 3072]\n",
      "Loading TF weight bert/encoder/layer_4/output/LayerNorm/beta with shape [768]\n",
      "Loading TF weight bert/encoder/layer_4/output/LayerNorm/gamma with shape [768]\n",
      "Loading TF weight bert/encoder/layer_4/output/dense/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_4/output/dense/kernel with shape [3072, 768]\n",
      "Loading TF weight bert/encoder/layer_5/attention/output/LayerNorm/beta with shape [768]\n",
      "Loading TF weight bert/encoder/layer_5/attention/output/LayerNorm/gamma with shape [768]\n",
      "Loading TF weight bert/encoder/layer_5/attention/output/dense/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_5/attention/output/dense/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_5/attention/self/key/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_5/attention/self/key/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_5/attention/self/query/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_5/attention/self/query/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_5/attention/self/value/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_5/attention/self/value/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_5/intermediate/dense/bias with shape [3072]\n",
      "Loading TF weight bert/encoder/layer_5/intermediate/dense/kernel with shape [768, 3072]\n",
      "Loading TF weight bert/encoder/layer_5/output/LayerNorm/beta with shape [768]\n",
      "Loading TF weight bert/encoder/layer_5/output/LayerNorm/gamma with shape [768]\n",
      "Loading TF weight bert/encoder/layer_5/output/dense/bias with shape [768]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading TF weight bert/encoder/layer_5/output/dense/kernel with shape [3072, 768]\n",
      "Loading TF weight bert/encoder/layer_6/attention/output/LayerNorm/beta with shape [768]\n",
      "Loading TF weight bert/encoder/layer_6/attention/output/LayerNorm/gamma with shape [768]\n",
      "Loading TF weight bert/encoder/layer_6/attention/output/dense/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_6/attention/output/dense/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_6/attention/self/key/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_6/attention/self/key/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_6/attention/self/query/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_6/attention/self/query/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_6/attention/self/value/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_6/attention/self/value/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_6/intermediate/dense/bias with shape [3072]\n",
      "Loading TF weight bert/encoder/layer_6/intermediate/dense/kernel with shape [768, 3072]\n",
      "Loading TF weight bert/encoder/layer_6/output/LayerNorm/beta with shape [768]\n",
      "Loading TF weight bert/encoder/layer_6/output/LayerNorm/gamma with shape [768]\n",
      "Loading TF weight bert/encoder/layer_6/output/dense/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_6/output/dense/kernel with shape [3072, 768]\n",
      "Loading TF weight bert/encoder/layer_7/attention/output/LayerNorm/beta with shape [768]\n",
      "Loading TF weight bert/encoder/layer_7/attention/output/LayerNorm/gamma with shape [768]\n",
      "Loading TF weight bert/encoder/layer_7/attention/output/dense/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_7/attention/output/dense/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_7/attention/self/key/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_7/attention/self/key/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_7/attention/self/query/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_7/attention/self/query/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_7/attention/self/value/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_7/attention/self/value/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_7/intermediate/dense/bias with shape [3072]\n",
      "Loading TF weight bert/encoder/layer_7/intermediate/dense/kernel with shape [768, 3072]\n",
      "Loading TF weight bert/encoder/layer_7/output/LayerNorm/beta with shape [768]\n",
      "Loading TF weight bert/encoder/layer_7/output/LayerNorm/gamma with shape [768]\n",
      "Loading TF weight bert/encoder/layer_7/output/dense/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_7/output/dense/kernel with shape [3072, 768]\n",
      "Loading TF weight bert/encoder/layer_8/attention/output/LayerNorm/beta with shape [768]\n",
      "Loading TF weight bert/encoder/layer_8/attention/output/LayerNorm/gamma with shape [768]\n",
      "Loading TF weight bert/encoder/layer_8/attention/output/dense/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_8/attention/output/dense/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_8/attention/self/key/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_8/attention/self/key/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_8/attention/self/query/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_8/attention/self/query/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_8/attention/self/value/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_8/attention/self/value/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_8/intermediate/dense/bias with shape [3072]\n",
      "Loading TF weight bert/encoder/layer_8/intermediate/dense/kernel with shape [768, 3072]\n",
      "Loading TF weight bert/encoder/layer_8/output/LayerNorm/beta with shape [768]\n",
      "Loading TF weight bert/encoder/layer_8/output/LayerNorm/gamma with shape [768]\n",
      "Loading TF weight bert/encoder/layer_8/output/dense/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_8/output/dense/kernel with shape [3072, 768]\n",
      "Loading TF weight bert/encoder/layer_9/attention/output/LayerNorm/beta with shape [768]\n",
      "Loading TF weight bert/encoder/layer_9/attention/output/LayerNorm/gamma with shape [768]\n",
      "Loading TF weight bert/encoder/layer_9/attention/output/dense/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_9/attention/output/dense/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_9/attention/self/key/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_9/attention/self/key/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_9/attention/self/query/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_9/attention/self/query/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_9/attention/self/value/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_9/attention/self/value/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_9/intermediate/dense/bias with shape [3072]\n",
      "Loading TF weight bert/encoder/layer_9/intermediate/dense/kernel with shape [768, 3072]\n",
      "Loading TF weight bert/encoder/layer_9/output/LayerNorm/beta with shape [768]\n",
      "Loading TF weight bert/encoder/layer_9/output/LayerNorm/gamma with shape [768]\n",
      "Loading TF weight bert/encoder/layer_9/output/dense/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_9/output/dense/kernel with shape [3072, 768]\n",
      "Loading TF weight bert/pooler/dense/bias with shape [768]\n",
      "Loading TF weight bert/pooler/dense/kernel with shape [768, 768]\n",
      "Loading TF weight cls/predictions/output_bias with shape [28996]\n",
      "Loading TF weight cls/predictions/transform/LayerNorm/beta with shape [768]\n",
      "Loading TF weight cls/predictions/transform/LayerNorm/gamma with shape [768]\n",
      "Loading TF weight cls/predictions/transform/dense/bias with shape [768]\n",
      "Loading TF weight cls/predictions/transform/dense/kernel with shape [768, 768]\n",
      "Loading TF weight cls/seq_relationship/output_bias with shape [2]\n",
      "Loading TF weight cls/seq_relationship/output_weights with shape [2, 768]\n"
     ]
    }
   ],
   "source": [
    "names = []\n",
    "arrays = []\n",
    "for name, shape in init_vars:\n",
    "    print(\"Loading TF weight {} with shape {}\".format(name, shape))\n",
    "    array = tf.train.load_variable(tf_path, name)\n",
    "    names.append(name)\n",
    "    arrays.append(array)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pytorch_pretrained_bert  import BertConfig, BertForPreTraining"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Building PyTorch model from configuration: {\n",
      "  \"attention_probs_dropout_prob\": 0.1,\n",
      "  \"hidden_act\": \"gelu\",\n",
      "  \"hidden_dropout_prob\": 0.1,\n",
      "  \"hidden_size\": 768,\n",
      "  \"initializer_range\": 0.02,\n",
      "  \"intermediate_size\": 3072,\n",
      "  \"max_position_embeddings\": 512,\n",
      "  \"num_attention_heads\": 12,\n",
      "  \"num_hidden_layers\": 12,\n",
      "  \"type_vocab_size\": 2,\n",
      "  \"vocab_size\": 28996\n",
      "}\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Initialise PyTorch model\n",
    "config = BertConfig.from_json_file('weights/pubmed_pmc_470k/bert_config.json')\n",
    "print(\"Building PyTorch model from configuration: {}\".format(str(config)))\n",
    "model = BertForPreTraining(config)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initialize PyTorch weight ['bert', 'embeddings', 'LayerNorm', 'beta']\n",
      "Initialize PyTorch weight ['bert', 'embeddings', 'LayerNorm', 'gamma']\n",
      "Initialize PyTorch weight ['bert', 'embeddings', 'position_embeddings']\n",
      "Initialize PyTorch weight ['bert', 'embeddings', 'token_type_embeddings']\n",
      "Initialize PyTorch weight ['bert', 'embeddings', 'word_embeddings']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_0', 'attention', 'output', 'LayerNorm', 'beta']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_0', 'attention', 'output', 'LayerNorm', 'gamma']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_0', 'attention', 'output', 'dense', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_0', 'attention', 'output', 'dense', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_0', 'attention', 'self', 'key', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_0', 'attention', 'self', 'key', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_0', 'attention', 'self', 'query', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_0', 'attention', 'self', 'query', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_0', 'attention', 'self', 'value', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_0', 'attention', 'self', 'value', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_0', 'intermediate', 'dense', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_0', 'intermediate', 'dense', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_0', 'output', 'LayerNorm', 'beta']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_0', 'output', 'LayerNorm', 'gamma']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_0', 'output', 'dense', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_0', 'output', 'dense', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_1', 'attention', 'output', 'LayerNorm', 'beta']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_1', 'attention', 'output', 'LayerNorm', 'gamma']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_1', 'attention', 'output', 'dense', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_1', 'attention', 'output', 'dense', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_1', 'attention', 'self', 'key', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_1', 'attention', 'self', 'key', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_1', 'attention', 'self', 'query', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_1', 'attention', 'self', 'query', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_1', 'attention', 'self', 'value', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_1', 'attention', 'self', 'value', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_1', 'intermediate', 'dense', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_1', 'intermediate', 'dense', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_1', 'output', 'LayerNorm', 'beta']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_1', 'output', 'LayerNorm', 'gamma']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_1', 'output', 'dense', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_1', 'output', 'dense', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_10', 'attention', 'output', 'LayerNorm', 'beta']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_10', 'attention', 'output', 'LayerNorm', 'gamma']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_10', 'attention', 'output', 'dense', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_10', 'attention', 'output', 'dense', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_10', 'attention', 'self', 'key', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_10', 'attention', 'self', 'key', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_10', 'attention', 'self', 'query', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_10', 'attention', 'self', 'query', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_10', 'attention', 'self', 'value', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_10', 'attention', 'self', 'value', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_10', 'intermediate', 'dense', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_10', 'intermediate', 'dense', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_10', 'output', 'LayerNorm', 'beta']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_10', 'output', 'LayerNorm', 'gamma']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_10', 'output', 'dense', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_10', 'output', 'dense', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_11', 'attention', 'output', 'LayerNorm', 'beta']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_11', 'attention', 'output', 'LayerNorm', 'gamma']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_11', 'attention', 'output', 'dense', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_11', 'attention', 'output', 'dense', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_11', 'attention', 'self', 'key', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_11', 'attention', 'self', 'key', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_11', 'attention', 'self', 'query', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_11', 'attention', 'self', 'query', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_11', 'attention', 'self', 'value', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_11', 'attention', 'self', 'value', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_11', 'intermediate', 'dense', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_11', 'intermediate', 'dense', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_11', 'output', 'LayerNorm', 'beta']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_11', 'output', 'LayerNorm', 'gamma']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_11', 'output', 'dense', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_11', 'output', 'dense', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_2', 'attention', 'output', 'LayerNorm', 'beta']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_2', 'attention', 'output', 'LayerNorm', 'gamma']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_2', 'attention', 'output', 'dense', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_2', 'attention', 'output', 'dense', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_2', 'attention', 'self', 'key', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_2', 'attention', 'self', 'key', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_2', 'attention', 'self', 'query', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_2', 'attention', 'self', 'query', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_2', 'attention', 'self', 'value', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_2', 'attention', 'self', 'value', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_2', 'intermediate', 'dense', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_2', 'intermediate', 'dense', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_2', 'output', 'LayerNorm', 'beta']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_2', 'output', 'LayerNorm', 'gamma']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_2', 'output', 'dense', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_2', 'output', 'dense', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_3', 'attention', 'output', 'LayerNorm', 'beta']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_3', 'attention', 'output', 'LayerNorm', 'gamma']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_3', 'attention', 'output', 'dense', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_3', 'attention', 'output', 'dense', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_3', 'attention', 'self', 'key', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_3', 'attention', 'self', 'key', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_3', 'attention', 'self', 'query', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_3', 'attention', 'self', 'query', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_3', 'attention', 'self', 'value', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_3', 'attention', 'self', 'value', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_3', 'intermediate', 'dense', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_3', 'intermediate', 'dense', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_3', 'output', 'LayerNorm', 'beta']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_3', 'output', 'LayerNorm', 'gamma']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_3', 'output', 'dense', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_3', 'output', 'dense', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_4', 'attention', 'output', 'LayerNorm', 'beta']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_4', 'attention', 'output', 'LayerNorm', 'gamma']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_4', 'attention', 'output', 'dense', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_4', 'attention', 'output', 'dense', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_4', 'attention', 'self', 'key', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_4', 'attention', 'self', 'key', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_4', 'attention', 'self', 'query', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_4', 'attention', 'self', 'query', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_4', 'attention', 'self', 'value', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_4', 'attention', 'self', 'value', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_4', 'intermediate', 'dense', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_4', 'intermediate', 'dense', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_4', 'output', 'LayerNorm', 'beta']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_4', 'output', 'LayerNorm', 'gamma']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_4', 'output', 'dense', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_4', 'output', 'dense', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_5', 'attention', 'output', 'LayerNorm', 'beta']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_5', 'attention', 'output', 'LayerNorm', 'gamma']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_5', 'attention', 'output', 'dense', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_5', 'attention', 'output', 'dense', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_5', 'attention', 'self', 'key', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_5', 'attention', 'self', 'key', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_5', 'attention', 'self', 'query', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_5', 'attention', 'self', 'query', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_5', 'attention', 'self', 'value', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_5', 'attention', 'self', 'value', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_5', 'intermediate', 'dense', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_5', 'intermediate', 'dense', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_5', 'output', 'LayerNorm', 'beta']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_5', 'output', 'LayerNorm', 'gamma']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_5', 'output', 'dense', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_5', 'output', 'dense', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_6', 'attention', 'output', 'LayerNorm', 'beta']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_6', 'attention', 'output', 'LayerNorm', 'gamma']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_6', 'attention', 'output', 'dense', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_6', 'attention', 'output', 'dense', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_6', 'attention', 'self', 'key', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_6', 'attention', 'self', 'key', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_6', 'attention', 'self', 'query', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_6', 'attention', 'self', 'query', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_6', 'attention', 'self', 'value', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_6', 'attention', 'self', 'value', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_6', 'intermediate', 'dense', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_6', 'intermediate', 'dense', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_6', 'output', 'LayerNorm', 'beta']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_6', 'output', 'LayerNorm', 'gamma']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_6', 'output', 'dense', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_6', 'output', 'dense', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_7', 'attention', 'output', 'LayerNorm', 'beta']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_7', 'attention', 'output', 'LayerNorm', 'gamma']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_7', 'attention', 'output', 'dense', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_7', 'attention', 'output', 'dense', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_7', 'attention', 'self', 'key', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_7', 'attention', 'self', 'key', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_7', 'attention', 'self', 'query', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_7', 'attention', 'self', 'query', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_7', 'attention', 'self', 'value', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_7', 'attention', 'self', 'value', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_7', 'intermediate', 'dense', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_7', 'intermediate', 'dense', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_7', 'output', 'LayerNorm', 'beta']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_7', 'output', 'LayerNorm', 'gamma']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_7', 'output', 'dense', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_7', 'output', 'dense', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_8', 'attention', 'output', 'LayerNorm', 'beta']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_8', 'attention', 'output', 'LayerNorm', 'gamma']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_8', 'attention', 'output', 'dense', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_8', 'attention', 'output', 'dense', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_8', 'attention', 'self', 'key', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_8', 'attention', 'self', 'key', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_8', 'attention', 'self', 'query', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_8', 'attention', 'self', 'query', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_8', 'attention', 'self', 'value', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_8', 'attention', 'self', 'value', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_8', 'intermediate', 'dense', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_8', 'intermediate', 'dense', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_8', 'output', 'LayerNorm', 'beta']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_8', 'output', 'LayerNorm', 'gamma']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_8', 'output', 'dense', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_8', 'output', 'dense', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_9', 'attention', 'output', 'LayerNorm', 'beta']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_9', 'attention', 'output', 'LayerNorm', 'gamma']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_9', 'attention', 'output', 'dense', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_9', 'attention', 'output', 'dense', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_9', 'attention', 'self', 'key', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_9', 'attention', 'self', 'key', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_9', 'attention', 'self', 'query', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_9', 'attention', 'self', 'query', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_9', 'attention', 'self', 'value', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_9', 'attention', 'self', 'value', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_9', 'intermediate', 'dense', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_9', 'intermediate', 'dense', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_9', 'output', 'LayerNorm', 'beta']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_9', 'output', 'LayerNorm', 'gamma']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_9', 'output', 'dense', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_9', 'output', 'dense', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'pooler', 'dense', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'pooler', 'dense', 'kernel']\n",
      "Initialize PyTorch weight ['cls', 'predictions', 'output_bias']\n",
      "Initialize PyTorch weight ['cls', 'predictions', 'transform', 'LayerNorm', 'beta']\n",
      "Initialize PyTorch weight ['cls', 'predictions', 'transform', 'LayerNorm', 'gamma']\n",
      "Initialize PyTorch weight ['cls', 'predictions', 'transform', 'dense', 'bias']\n",
      "Initialize PyTorch weight ['cls', 'predictions', 'transform', 'dense', 'kernel']\n",
      "Initialize PyTorch weight ['cls', 'seq_relationship', 'output_bias']\n",
      "Initialize PyTorch weight ['cls', 'seq_relationship', 'output_weights']\n"
     ]
    },
    {
     "ename": "NameError",
     "evalue": "name 'pytorch_dump_path' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-31-b321710b346a>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     37\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     38\u001b[0m \u001b[0;31m# Save pytorch-model\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 39\u001b[0;31m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Save PyTorch model to {}\"\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpytorch_dump_path\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     40\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msave\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstate_dict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpytorch_dump_path\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'pytorch_dump_path' is not defined"
     ]
    }
   ],
   "source": [
    "\n",
    "for name, array in zip(names, arrays):\n",
    "    name = name.split('/')\n",
    "    # adam_v and adam_m are variables used in AdamWeightDecayOptimizer to calculated m and v\n",
    "    # which are not required for using pretrained model\n",
    "    if any(n in [\"adam_v\", \"adam_m\", \"global_step\"] for n in name):\n",
    "        print(\"Skipping {}\".format(\"/\".join(name)))\n",
    "        continue\n",
    "    pointer = model\n",
    "    for m_name in name:\n",
    "        if re.fullmatch(r'[A-Za-z]+_\\d+', m_name):\n",
    "            l = re.split(r'_(\\d+)', m_name)\n",
    "        else:\n",
    "            l = [m_name]\n",
    "        if l[0] == 'kernel' or l[0] == 'gamma':\n",
    "            pointer = getattr(pointer, 'weight')\n",
    "        elif l[0] == 'output_bias' or l[0] == 'beta':\n",
    "            pointer = getattr(pointer, 'bias')\n",
    "        elif l[0] == 'output_weights':\n",
    "            pointer = getattr(pointer, 'weight')\n",
    "        else:\n",
    "            pointer = getattr(pointer, l[0])\n",
    "        if len(l) >= 2:\n",
    "            num = int(l[1])\n",
    "            pointer = pointer[num]\n",
    "    if m_name[-11:] == '_embeddings':\n",
    "        pointer = getattr(pointer, 'weight')\n",
    "    elif m_name == 'kernel':\n",
    "        array = np.transpose(array)\n",
    "    try:\n",
    "        assert pointer.shape == array.shape\n",
    "    except AssertionError as e:\n",
    "        e.args += (pointer.shape, array.shape)\n",
    "        raise\n",
    "    print(\"Initialize PyTorch weight {}\".format(name))\n",
    "    pointer.data = torch.from_numpy(array)\n",
    "\n",
    "# Save pytorch-model\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Save PyTorch model to weights/\n"
     ]
    }
   ],
   "source": [
    "print(\"Save PyTorch model to {}\".format('weights/'))\n",
    "torch.save(model.state_dict(),'weights/pytorch_weight')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import re\n",
    "import argparse\n",
    "import tensorflow as tf\n",
    "import torch\n",
    "import numpy as np\n",
    "\n",
    "from pytorch_pretrained_bert import BertConfig, BertForPreTraining\n",
    "\n",
    "def convert_tf_checkpoint_to_pytorch(tf_checkpoint_path, bert_config_file, pytorch_dump_path):\n",
    "    config_path = os.path.abspath(bert_config_file)\n",
    "    tf_path = os.path.abspath(tf_checkpoint_path)\n",
    "    print(\"Converting TensorFlow checkpoint from {} with config at {}\".format(tf_path, config_path))\n",
    "    # Load weights from TF model\n",
    "    init_vars = tf.train.list_variables(tf_path)\n",
    "    names = []\n",
    "    arrays = []\n",
    "    for name, shape in init_vars:\n",
    "        print(\"Loading TF weight {} with shape {}\".format(name, shape))\n",
    "        array = tf.train.load_variable(tf_path, name)\n",
    "        names.append(name)\n",
    "        arrays.append(array)\n",
    "\n",
    "    # Initialise PyTorch model\n",
    "    config = BertConfig.from_json_file(bert_config_file)\n",
    "    print(\"Building PyTorch model from configuration: {}\".format(str(config)))\n",
    "    model = BertForPreTraining(config)\n",
    "\n",
    "    for name, array in zip(names, arrays):\n",
    "        name = name.split('/')\n",
    "        # adam_v and adam_m are variables used in AdamWeightDecayOptimizer to calculated m and v\n",
    "        # which are not required for using pretrained model\n",
    "        if any(n in [\"adam_v\", \"adam_m\", \"global_step\"] for n in name):\n",
    "            print(\"Skipping {}\".format(\"/\".join(name)))\n",
    "            continue\n",
    "        pointer = model\n",
    "        for m_name in name:\n",
    "            if re.fullmatch(r'[A-Za-z]+_\\d+', m_name):\n",
    "                l = re.split(r'_(\\d+)', m_name)\n",
    "            else:\n",
    "                l = [m_name]\n",
    "            if l[0] == 'kernel' or l[0] == 'gamma':\n",
    "                pointer = getattr(pointer, 'weight')\n",
    "            elif l[0] == 'output_bias' or l[0] == 'beta':\n",
    "                pointer = getattr(pointer, 'bias')\n",
    "            elif l[0] == 'output_weights':\n",
    "                pointer = getattr(pointer, 'weight')\n",
    "            else:\n",
    "                pointer = getattr(pointer, l[0])\n",
    "            if len(l) >= 2:\n",
    "                num = int(l[1])\n",
    "                pointer = pointer[num]\n",
    "        if m_name[-11:] == '_embeddings':\n",
    "            pointer = getattr(pointer, 'weight')\n",
    "        elif m_name == 'kernel':\n",
    "            array = np.transpose(array)\n",
    "        try:\n",
    "            assert pointer.shape == array.shape\n",
    "        except AssertionError as e:\n",
    "            e.args += (pointer.shape, array.shape)\n",
    "            raise\n",
    "        print(\"Initialize PyTorch weight {}\".format(name))\n",
    "        pointer.data = torch.from_numpy(array)\n",
    "\n",
    "    # Save pytorch-model\n",
    "    print(\"Save PyTorch model to {}\".format(pytorch_dump_path))\n",
    "    torch.save(model.state_dict(), pytorch_dump_path)\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
